# coding=utf-8

import numpy
import redis

import config.setting as setting

similarity_simhash_namespace = 'sim_hash:'
sim_distance = 1


def calc_duplicate(origin_id, content_vec, url, r):
    '''
    计算重复
    :param origin_id:
    :param content_vec:
    :return:如果没有重复，返回None，如果有，返回重复文章的信息 int(hash4_redis_value_file_id), hash4_redis_value_file_url, dis4
    '''
    if content_vec is None or len(content_vec) == 0:
        return None
    content_vec_list = []
    for item in content_vec.split(' '):
        content_vec_list.append(item.split('/'))

    hash = get_hash(content_vec_list)
    # 按照hash桶数据结构，提高查找效率
    hash_seg_list = list()
    for i in range(4):
        start = i * 16
        end = start + 16
        hash_seg_list.append(hash[start:end])

    for hash_seg in hash_seg_list:
        if r.exists(setting.NAME_SPACE + similarity_simhash_namespace + hash_seg):
            # lrange的结果是byte类型
            # 保存[hash, origin_id, url]
            saved_values_list = r.lrange(setting.NAME_SPACE + similarity_simhash_namespace + hash_seg, 0, -1)
            for saved_values in saved_values_list:
                saved_hash, saved_id, saved_url, saved_content = saved_values.decode().split(',')
                dis = hamming_dis(hash, saved_hash)
                if dis <= sim_distance:
                    if int(saved_id) >= int(origin_id):  # 新来的id必须是大的，否则可能是重跑过，这种情况过滤一下
                        continue
                    return saved_id, saved_url, dis, saved_content

    values = '%s,%d,%s,%s' % (hash, origin_id, url, '')
    for hash_seg in hash_seg_list:
        if values not in dict.fromkeys(
                r.lrange(setting.NAME_SPACE + similarity_simhash_namespace + hash_seg, 0, -1)):
            r.rpush(setting.NAME_SPACE + similarity_simhash_namespace + hash_seg, values)
            r.expire(setting.NAME_SPACE + similarity_simhash_namespace + hash_seg, setting.REDIS_EXPIRE_TIME)

    return None


# get hamming distance for content1 and conten2
def get_sim_distance(content1, content2):
    '''
    获取simhash的海明距离
    :param content1:
    :param content2:
    :return:
    '''
    hash1 = get_hash(content1)
    hash2 = get_hash(content2)
    return hamming_dis(hash1, hash2)


def get_hash(feature_weight):
    # # 即先按照权重排序，再按照词排序
    key_list = []
    # print(feature_weight)
    for feature, weight in feature_weight:
        weight = int(float(weight) * len(feature_weight))
        feature = string_hash(feature)
        # print feature
        temp = []
        for i in feature:
            if i == '1':
                temp.append(weight)
            else:
                temp.append(-weight)
                # print(temp)
        key_list.append(temp)
        # print key_list
    list1 = numpy.sum(numpy.array(key_list), axis=0)
    # print(list1)
    if len(key_list) == 0:  # 编码读不出来
        return '00'
    simhash = ''
    for i in list1:
        if i > 0:
            simhash = simhash + '1'
        else:
            simhash = simhash + '0'
    return simhash


def string_hash(source):
    if source == "":
        return 0
    else:
        x = ord(source[0]) << 7
        m = 1000003
        mask = 2 ** 128 - 1
        for c in source:
            x = ((x * m) ^ ord(c)) & mask
        x ^= len(source)
        if x == -1:
            x = -2
        x = bin(x).replace('0b', '').zfill(64)[-64:]
        # print source, x
        return str(x)


def hamming_dis(hash1, hash2):
    t1 = '0b' + hash1
    t2 = '0b' + hash2
    n = int(t1, 2) ^ int(t2, 2)
    i = 0
    while n:
        n &= (n - 1)
        i += 1
    return i


if __name__ == '__main__':
    # 清除一下测试用的redis
    r = redis.Redis(host="172.16.202.40",
                    port=6379,
                    db=7)

    # content 格式：
    # 取文章全部词
    # 列表中放元组，元组内容为关键词和tfidf
    content = [('北京', 0.42960629744747253), ('北京', 0.3284276786510989), ('俄罗斯', 0.32440333982736264),
               ('世预赛', 0.2933832370057143)]
    id = 1
    url = 'http://www.sina.com'
    # print similarityObj.calcDuplicate(id, url, content)

    content2 = [('北京', 0.42960629744747253), ('北京', 0.3284276786510989), ('俄罗斯', 0.32440333982736264),
                ('世预赛', 0.2933832370057143)]
    id2 = 1
    url2 = 'http://www.sina.com'

    # print(get_sim_distance(content, content2))

    origin_id = 1234578
    content_vec = '小兰/0.8901340481896628 四个/0.19058218899509366 怀孕/0.1245109606937783 可是/0.12331710338409275 婆婆/0.09118536941675928 推三阻四/0.08092127710815117 检查/0.08022326724499078 没太大/0.07574125987397032 偏瘦/0.07075210262867708 胸腔/0.06524726675050631 单子/0.06514147838175963 最后/0.06272552601145234 看得出来/0.06071239014086313 泪流满面/0.05893824338490224 肚子/0.05855507268726102 不敢相信/0.058164279004747445 遭罪/0.05722373644622201 之后/0.056988591039729614 明显/0.05690534212185644 里面/0.05515287877537832 动静/0.05187282731486796 不对劲/0.051289555244339105 拉着/0.05034969541051627 吓坏/0.049535422886842616 一说/0.049515956251878906 却说/0.04653293768661727 多月/0.04650480276673884 缓慢/0.045557680557215666 医生/0.045473515879440665 整体/0.0449563257390487 还是/0.04177941810011874 过得/0.04119922605856331 各种各样/0.04036680114907528 流产/0.0400887680183791 小男孩/0.03944452694164734 虽然/0.038736497269039966 比较/0.03719005811438675 看电视/0.03658235321595843 之外/0.035606615248058304 仍然/0.035380056618904185 放心/0.03504010650852332 子宫/0.03423741009052333 时候/0.034113606351579825 发现/0.033937791183474404 上班/0.033071171098362113 全部/0.032512523132237105 已经/0.032457046245619785 没有/0.03245691469810065 觉得/0.03206857853139038 孕妇/0.030622147243933775 好像/0.03008493263906785 结婚/0.03005875439905866 舒服/0.029912573150617308 本来/0.029438367677708015 赶紧/0.02913879641211565 家人/0.027708231107600505 下来/0.02704150485773598 有点/0.026698012948911016 睡觉/0.026180510578166546 所以/0.025641240410814947 吃饭/0.025355582544267168 老公/0.025236253689842714 照顾/0.02501726481316324 开心/0.024184669618132804 更加/0.024168305575021712 并且/0.024104386093183917 医院/0.023442891590933317 发育/0.023278078352256436 除了/0.02289980316354278 建议/0.022409673210695327 想要/0.021975899336479657 儿子/0.02188307800583091 结果/0.021785343157249948 而是/0.021614842522326077 这是/0.02127561941050174 来说/0.02030183458609983 能够/0.02000997557086027 一点/0.01935478045113627 选择/0.019297386751412047 出来/0.01921232611255846 一直/0.019106177982699177 每天/0.018698228962306697 身体/0.018687787470490387 非常/0.018652374242704158 自己/0.018199049721410666 知道/0.013392310270260156 可以/0.0100160639044666 一个/0.009658035453651768 孩子/0.007772624534451324'
    print(calc_duplicate(origin_id, content_vec, "http://www.sina.com", r))
